// 爬Ok资源采集
let cheerio = require('cheerio');
let superagent = require('superagent');
let async = require('async');
let express = require('express');
let MongoClient = require('mongodb').MongoClient;

let app = express();
let allDetailUrl = [];
let allData = [];
let index = 0;
let allUrl = [];
// 获取所有要爬的页面
for (let i = 1; i <= 1; i++) {
    allUrl.push(`http://www.okokzy.com/?m=vod-type-id-1-pg-${i}.html`)
}

const dbUrl = 'mongodb://localhost:27017/movie';
let db = MongoClient.connect(dbUrl, (err, db) => {
    console.log('连接数据库成功');
    main(db);
})

function main(db) {
    async.mapLimit(allUrl, 1, (itemUrl, callback) => {
        superagent.get(itemUrl)
            .end((err, sres) => {
                if (err) {
                    console.log('爬取首页页面出错');
                    return;
                }
                let $ = cheerio.load(sres.text);
                $('.xing_vb ul').each((idx, ele) => {
                    let $element = $(ele);
                    let name = $element.find('.xing_vb4').text();
                    if (name.length !== 0) {
                        let href = `http://www.okokzy.com${$element.find('.xing_vb4 a').attr('href')}`;
                        console.log(`获取到链接: ${href}`);
                        allDetailUrl.push({
                            href,
                        })
                    }
                })
                callback(null, `请求成功${url}`);
            })
    }, (err, result) => {
        console.log('爬虫成功');
        getDetail(db);
    })
}

function getDetail(db) {
    async.mapLimit(allDetailUrl, 1, (item, callback) => {
        superagent.get(item.href)
            .end((err, sres) => {
                if (err) {
                    console.log("抓取详情页发生错误" + err);
                    return;
                }
                let $ = cheerio.load(sres.text);
                let data = $('input[name=copy_sel]').eq(1).attr('value');
                if (data != null) {
                    let detailItem = {
                        movieId: ++index,
                        name: $('.vodh').find('h2').text(),
                        img: $('.lazy').attr('src'),
                        src: data,
                    }
                    db.collection('movies').insert(detailItem, (err, res) => {
                        if (err) {
                            console.log('添加数据库出错');
                            return;
                        }
                        console.log('添加数据成功');
                    });
                }

                console.log('爬到一条数据~~~~~~')
                callback(null, '请求成功');
            })
    }, (err, result) => {
        console.log(allData);
        console.log('爬虫成功--------------------------');
        return allData;
    })
}

// app.get('/', (req, res) => {
//     main();
// })

// app.listen(4000, () => {
//     console.log('app is listening at port 4000');
// })